# Init db
initRedis()
# redisFlushDB()
redisKeys()


author <- "Kris Gevaert"
from <- 2003
to <- 2011
journals <- c(pubmed.get.Journals.by.Author.Date(author=author, from=from, to=to))
years <- from:to
# 
# journal <- "Proteomics"
# year <- 2005
# journal <- journals[6]

for(journal in journals){
          for (year in years){
                    
                    journal.key <- transformJournalForRedis(journal)
                    
                    redis.key.citation <- makeRedisCitationKey(journal, year)
                    redis.key.pubmedid <- makeRedisPIDKey(journal, year)
                    
                    initRedis()
                    if(redisExists(redis.key.citation)){
                              cat(sprintf("redis key %s exists. Proceeding to next key.\n", redis.key.citation))
                    }else{
                              
                              tryCatch({
                                        cat(sprintf("loading %s in %s\n", journal, year))
                                        data.pubmed <- pubmed.get.df.by.Journal(journal=journal, from=year, to=year)
                                        data.gs <- gs.get.all.by.journal(journal=journal,from=year,to=year)
                                        merge <- pubgs(data.pubmed=data.pubmed, data.gs=data.gs)
                                        
                                        match.rate <- nrow(merge) / nrow(data.pubmed)
                                        if(match.rate < 0.6){
                                                  sprintf("skipping '%s' because of the low matching rate (%s)!\n", redis.key.citation, round(match.rate, 2))
                                                  next
                                        }
                                        
                                        initRedis()
                                        for(i in 1:nrow(merge)){
                                                  redisLPush(key=redis.key.citation, merge$citation_gs[i])
                                                  redisLPush(key=redis.key.pubmedid, merge$pid[i])
                                        }
                                        redisSave()
                                        redisClose()
                                        
                                        cat(sprintf("added %s entries in %s\n", nrow(merge), redis.key.citation))
                                        
                              }, interrupt = function(ex) {
                                        cat("An interrupt was detected.\n");
                                        print(ex);
                              }, error = function(ex) {
                                        cat("An error was detected.\n");
                                        print(ex);
                              }, finally = {
                                        cat("done\n");
                                        if(is.null(dev.list()) == FALSE){
                                                  dev.off()
                                        }
                              })
                    }
                    
                    sleep.time <- 66
                    cat(sprintf("sleeping for %s seconds after %s\n", sleep.time, redis.key.citation))
                    Sys.sleep(sleep.time)
                    
          }
          
          sleep.time <- 166
          cat(sprintf("sleeping for %s seconds after journal %s\n", sleep.time, journal))
          Sys.sleep(sleep.time)
          
}

# Init db
initRedis()
# redisFlushDB()
redisKeys()



